import fileinput
import sys
import time
import cPickle

#CREATE HOURLY SAMPLES FROM LOGS


DOMAIN_NAME = 0
CLIENT_IP = 1                  
COMPLETION_TIME = 3    
HTTP_REQUEST = 4       
STATUS_CODE = 5
RETURNED_SIZE = 6
REFERRER = 7
USER_AGENT = 8
dic={}




def hashing():
    start = time.clock()
    count =0
    totalcount = 0
    jpgcount = 0
    pngcount = 0
    csscount = 0
    gifcount =0
    unsuccesscount =0
    nonjpgcount = 0
    cleaned_count=0
    domains = {}
    ad_hash = {}
    rest = main()
    
    
    outfile1 = open('D:\\Access Logs/access_log.18/seveneight2.txt', "w")
    outfile2 = open('D:\\Access Logs/access_log.18/eightnine2.txt', "w")
    
    for line in fileinput.input(['D:\\Access Logs/access_log.18/adservers.txt']):
        try:
            #print line
            line1=line.split()[0]
            ad_hash[line1]="*"
        except:
            continue
        
    for line in fileinput.input(['D:\\Access Logs/access_log.20/access_log.20']):
        try:         
            row = getlogfields(line)
            expandedrow = expandedfields(row)
            fields = expandedrow[2].split(".")
            totalcount+=1
            length = len(fields)
#            if fields[length-1].startswith("jpg") or fields[length-1].startswith("png") or fields[length-1].startswith("gif") or fields[length-1].startswith("css"):
            if fields[length-1]=="jpg": 
                jpgcount +=1
                continue
            elif fields[length-1]=="png":
                pngcount +=1
                continue
            elif fields[length-1]=="gif":
                gifcount +=1
                continue
            elif fields[length-1]=="css":
                csscount +=1
                continue
            elif int(row[5])<200 or int(row[5])>299:
                unsuccesscount +=1
                continue
            else:
                nonjpgcount+=1
                if ad_hash.has_key(row[0]):
                    continue
                cleaned_count+=1
                domainname = lookup(row[0], rest)
                if not (len(domainname.split('.')[1])>0):
                    continue
                domainnumber = domainmapper(domainname, domains)
                log = str(domainnumber)+" "+expandedrow[0]+" "+row[2]+" "+expandedrow[1]+" \n"
                
            if expandedrow[6] == "07":
                outfile1.write(log)
            elif expandedrow[6] == "08":
                outfile2.write(log)
            elif expandedrow[6] == "09":
                break

        except:
            continue
    outfile1.close()
    print "Totalcount : ",totalcount
    print "Jpgcount : ", jpgcount
    print "Pngcount : ", pngcount
    print "Csscount : ", csscount
    print "GifCount : ", gifcount
    print "Unsuccess access : ", unsuccesscount
    print "NonJpg : ", nonjpgcount
    print "Cleaned Count  : ", cleaned_count
    
    
    print "DICTIONARY  "
    print len(domains)
    print domains.items()
    print "Sum : ", jpgcount+nonjpgcount

    
    
    
#    obj = cPickle.load(open('D:\\save13.p', 'rb'))
#    print "got it back"
#    for key in obj.keys():
#        value=dic.get(key)
#        print value.values()
    elapsed = (time.clock() - start)
    print elapsed
    print count
    sys.exit()
                    


def domainmapper(domain,domains):
    if domains.has_key(domain):
        return domains.get(domain)
    else:
        size = len(domains)
        domains[domain]= size+1
        return size+1
    
def test():
    print "Success testing "
#    print dic.values()
    if dic.has_key("861ee80996964b5c1a01a8d34a2f31ed"):
        free = dic.get("861ee80996964b5c1a01a8d34a2f31ed")
#        print "Codeeeeeeeeeeee", free.items() 
    #cPickle.dump(dic, open('D:\\save13.p', 'wb'))
         
              

#class Logrecord:
#    Hostname, IP, UserId,Time, Request, Response, Downloadvol, Referrer, Useragent
#    def __init__(self):
#        self.hostname = 0
#        self.ip = 0
#        self.userid = 0
#        self.time = 0
#        self.request = 0
#        self.response = 0
#        self.download = 0      

def getlogfields(s):
    fields = []
    # error! no leading spaces allowed!
    if s.startswith(" "): 
       return fields

    state = 0
    quoteds = ""
    n = 0
    for f in s.split():
        if state == 0:
           if f.startswith("\""):
               if f.endswith("\""):
                   fields.append(f[1:-1])
               else:
                   quoteds = f[1:] + " "
                   state = 2
               continue
           elif f.startswith("'"):
               if f.endswith("\""):
                   fields.append(f[1:-1])
               else:
                   quoteds = f[1:] + " "
                   state = 3
               continue
           elif f.startswith("["):
               if f.endswith("]"):
                   fields.append(f[1:-1])
               else:
                   blockeds = f[1:] + " "
                   state = 4
               continue
           else:
               fields.append(f)
 
        if state == 2 :
            quoteds += (f + " ")
            if f.endswith("\"") :
               quoteds = quoteds[:-2]
               fields.append(quoteds)
               n += 1
               state = 0
 
        if state == 3 :
            quoteds += (f + " ")
            if f.endswith("'") :
               quoteds = quoteds[:-2]
               fields.append(quoteds)
               n += 1
               state = 0
 
        if state == 4 :
            blockeds += (f + " ") 
            if f.endswith("]") :
               blockeds = blockeds[:-2]
               fields.append(blockeds)
               n += 1
               state = 0
 
        #@@@ print "processing [", f, "]; state=", state,";current=",  fields, "quoteds=", quoteds
    return fields

def logsplittimefield(timefield):
    try:
#        print "debugging...", timefield
        (stime, zone) = timefield.split()
        caldate, hour, minute, second = stime[0:].split(":")
        day, month, year = caldate.split("/")
    except:
#        print "error procedding timefield [%s]\n" % timefield
        return None, None, None, None, None, None, None
    return (day, month, year, hour, minute, second, zone)

def logsplitrequestfield(requestfield):
    try:
        # you can avoid the intermediate storing to variables.
        method, resource, protocol = requestfield.split()
    except:
        return None, None, None
 
    return method, resource, protocol
 
 
def logsplituseragentfield(useragentfield):
    """
    Returns (browser, url)
    """
    return useragentfield.split()

def expandedfields(ninefields):
    """
    Returns a full tuple of information.
    """
    try:
        if len(ninefields) == 0:
            return None, None
        ip1, ip2 = ninefields[CLIENT_IP].split(",")
        if ip1 == '-':
            ip = ip2
        else:
            ip = ip1
        day, month, year, hour, minute, second, zone = logsplittimefield(ninefields[COMPLETION_TIME])
        timestamp = year + "-" + getmonth(month) + "-" + day + " " + hour + ":" + minute + ":" + second
        method, resource, protocol = logsplitrequestfield(ninefields[HTTP_REQUEST])
        status = ninefields[STATUS_CODE]
        size = ninefields[RETURNED_SIZE]
#        for i, f in enumerate(ninefields):
#            print i, f
        browserandurls = ninefields[USER_AGENT].split()
        browser = browserandurls[0]
        if len(browserandurls) == 1:
            urls = []
        else:
            urls = browserandurls[1:]
            urls[0] = urls[0][1:]
            urls[-1] = urls[-1][:-1]

        return (ip, timestamp,resource, day, month, year, hour, minute, second, zone, method, protocol, browser, urls)
    except:
#        print "error in processing [%s]" % ninefields
        return None, None, None

def getmonth(month):
    if month == "Jan":
        return "01"
    elif month == "Feb":
        return "02"
    elif month == "Mar":
        return "03"
    elif month == "Apr":
        return "04"
    elif month == "May":
        return "05"
    elif month == "Jun":
        return "06"
    elif month == "Jul":
        return "07"
    elif month == "Aug":
        return "08"
    elif month == "Sep":
        return "09"
    elif month == "Oct":
        return "10"
    elif month == "Nov":
        return "11"
    elif month == "Dec":
        return "12"
    else:
        return "00"


#import cPickle
#
#print "got it back"
#obj = cPickle.load(open('D:\\save.p', 'rb'))
#print "got it back"
#for key in obj.keys():
#    value=obj.get(key)
#    print value.values()

#!/usr/bin/env python2.5



def find_node(parent, parts):
    if not parts:
        return parent

    if len(parent) == 1:
        parent.append({})

    assert len(parent) == 2
    negate, children = parent

    child = parts.pop()
    try:
        child = child.encode('ascii')
    except:
        pass

    child_node = children.get(child, None)

    if not child_node:
        children[child] = child_node = [0]

    return find_node(child_node, parts)

def add_rule(root, rule):
    if rule.startswith('!'):
        negate = 1
        rule = rule[1:]
    else:
        negate = 0

    parts = rule.split('.')
    find_node(root, parts)[0] = negate

def simplify(node):
    if len(node) == 1:
        return node[0]

    return (node[0], dict((k, simplify(v))
                          for (k, v) in node[1].iteritems()))

def mini_pformat(o):
    if o in (0, 1):
        return str(o)
    elif type(o) in (str, unicode):
        return repr(o)
    elif type(o) is dict:
        return '{' + ','.join((mini_pformat(k)+':'+mini_pformat(v))
                              for (k, v) in o.iteritems()) + '}'
    else:
        assert type(o) == tuple
        if len(o) == 1:
            return '(%s,)' % mini_pformat(o2[0])
        else:
            return '(' + ','.join(mini_pformat(o2) for o2 in o) + ')'


def build_structure(fp):
    root = [0]

    for line in fp:
        line = line.decode('utf-8').strip()
        if line.startswith('//') or not line:
            continue

        add_rule(root, line.split()[0])
#    add_rule(root, "www.foo.bar.baz.com")

    return root


def main():
    root = build_structure(file('D:\\Access Logs/access_log.18/effective_tld_names.txt'))
    root = simplify(root)
    write_module(root)
    return root

def write_module(root):
    print '#!/usr/bin/env python'
    print
    print 'root =', mini_pformat(root)
    print
    print 

def lookup_node(matches, depth, parent, parts):
    if parent in (0, 1):
        negate = parent
        children = None
    else:
        negate, children = parent

    matches[-depth] = matches[-depth] or negate

    if depth < len(parts) and children:
        for name in (parts[-depth], '*'):
            child = children.get(name, None)
            if child is not None:
                lookup_node(matches, depth+1, child, parts)

def lookup(domain,root):
    parts = domain.split('.')
    hits = [None] * len(parts)

    lookup_node(hits, 1, root, parts)

    for i, what in enumerate(hits):
        if what is not None:
            return '.'.join(parts[i:])


#
#if __name__ == '__main__':
#    main()











if __name__ == "__main__": hashing()